package cn.com.ccxe.core.crawl.listing;

import java.io.IOException;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import cn.com.ccxe.core.crawl.CrawlExpressionCache;
import cn.com.ccxe.core.crawl.Fetcher;
import cn.com.ccxe.core.db.ListPageDB;
import cn.com.ccxe.core.db.ListPageWorkQueues;
import cn.com.ccxe.core.db.WebDB;
import cn.com.ccxe.core.entity.ArticleLink;
import cn.com.ccxe.core.entity.CrawlExpression;
import cn.com.ccxe.core.entity.ListPageLink;
import cn.com.ccxe.core.entity.WebData;
import cn.com.ccxe.core.util.CodeUtil;
import cn.com.ccxe.core.util.Configurations;
import cn.com.ccxe.dao.CrawlExpressionDao;
import cn.com.ccxe.dao.impl.CrawlExpressionDaoImpl;

public class ListPageCrawler implements Runnable {

	final static Logger 	logger = LoggerFactory.getLogger(ListPageCrawler.class);

	/** 重复抓取计数器 */
	private int 			refetchCount = 0;
	/** 重复抓取的次数 */
	private static int 		REFETCH_AMOUNT = Configurations.getInt("refetch.amount", 2);
	/** 处理成功的任务数 */
	private volatile int 	successTaskNum = 0;
	/** 处理失败的任务数 */
	private volatile int 	failedTaskNum = 0;
	/** 重启的任务数 */
	private volatile int 	restartTaskNum = 0;
	/** 线程名 */
	private String 			name;

	private String 			state;
	private AtomicBoolean 	crawlerSwitch = new AtomicBoolean(true);

	public ListPageCrawler(String name) {
		this.name = name;
		this.state = "NEW";
	}

	public void run() {
		CrawlExpressionDao listPageDao = new CrawlExpressionDaoImpl();
		final ListPageWorkQueues workQueue = ListPageWorkQueues.getInstance();
		final ListPageDB listPageDB = ListPageDB.getInstance();
		if (listPageDao == null)
			throw new NullPointerException("ListPageCrawlRuleDao is Null!");
		while (crawlerSwitch.get()) {
			try {
				this.state = "RUN";
				if (workQueue.size() <= 0) {
					Thread.sleep(10 * 1000);
					this.state = "SLEEP";
				}
				ListPageLink link = workQueue.consume();
				if (link == null) {
					continue;
				}
				CrawlExpression listPage = CrawlExpressionCache.get(link.getModuleId());
				link.setListPage(listPage);
				if (link.getListPage() == null) {
					logger.debug("ListPageLink:{}", link);
					logger.info("没有{}找到抽取规则，请设定！URL为{}", link.getTitle(), link.getPath());
					continue;
				}
				process(link);
				link.setFetchTime(CodeUtil.getDate());
				listPageDB.update(link.getPath(), link);
				successTaskNum++;

			} catch (Exception e) {
				// TODO 需确保线程存活。
				failedTaskNum++;
				e.printStackTrace();
			}
		}
	}

	/**
	 * 处理网页并进行归类。
	 * 
	 * @param data
	 *            网页数据。
	 */
	public void process(ListPageLink link) {

		Fetcher fetcher = new Fetcher();
		try {
			WebData data;
			data = fetcher.fetch(link);
			if (data.getHtml() == null && data.getHtml().length() <= 0) {
				logger.debug("采集{}时，没有发现内容！", link.getPath());
				return;
			}
			extractor(link, data);
		} catch (IOException e) {
			failedTaskNum++;
			e.printStackTrace();
			// 如果失败将再进行抓取，但次数不能超过两次。
			if (refetchCount < REFETCH_AMOUNT) {
				restartTaskNum++;
				process(link);
			}
		}
		refetchCount = 0;
	}

	private void extractor(ListPageLink link, WebData data) {
		ContentExtractor extractor = new ContentExtractor(data);
		Set<ArticleLink> articlLinks = extractor.extractorArticleList(link.getListPage().getArticleListExpre());
		Set<ListPageLink> listPageLinks = extractor.extractorListPage(link.getListPage().getListPageExpre());
		if (logger.isDebugEnabled()) {
			logger.debug("\n发现文章链接【{}】个、分页列表链接【{}】个,来自{}。", new Object[] { articlLinks.size(), listPageLinks.size(),
					link.getPath() });
		}

		saveListPageLink(link, listPageLinks);
		saveArticleLink(link, articlLinks);
	}

	private void saveArticleLink(ListPageLink link, Set<ArticleLink> articlLinks) {
		WebDB webDB = WebDB.getInstance();
		for (ArticleLink childLink : articlLinks) {
			childLink.setModuleId(link.getModuleId());
			webDB.put(childLink.getPath(), childLink);
		}
		logger.debug("来自 {} 的文章链接全部保存完毕！URL为：{}", link.getTitle(), link.getPath());
		webDB.sync();
	}

	private void saveListPageLink(ListPageLink link, Set<ListPageLink> listPageLinks) {
		ListPageDB listDB = ListPageDB.getInstance();
		for (ListPageLink childLink : listPageLinks) {
			childLink.setModuleId(link.getModuleId());
			childLink.setListPage(link.getListPage());
			childLink.setTitle(link.getTitle());
			listDB.put(childLink.getPath(), childLink);
		}
		logger.debug("来自 {} 的列表链接全部保存完毕！URL为：{}", link.getTitle(), link.getPath());
		listDB.sync();
	}

	public int getRestartTaskNum() {
		return restartTaskNum;
	}

	public int getFailedTaskNum() {
		return failedTaskNum;
	}

	public int getSuccessTaskNum() {
		return successTaskNum;
	}

	public void getCrawlerInfo() {
		logger.info("处理成功的任务数:{}", successTaskNum);
		logger.info("处理失败的任务数:{}", failedTaskNum);
		logger.info("重启的任务数:{}", restartTaskNum);
	}

	public String state() {
		return state;
	}

	public String name() {
		return name;
	}

	public void shutDown() {
		crawlerSwitch.set(false);
	}
}
